# -*- coding: utf-8 -*-
from scrapy.selector import Selector
import scrapy
import json
from scrapy.loader import ItemLoader, Identity
from spiders.items import OkexItem
from bs4 import BeautifulSoup
import time
from urllib import parse

class OkexSpider(scrapy.Spider):
    # 爬虫名称
    name = "okex"
    # 设置允许的域名
    allowed_domains = ["okex.com"]
    # 设置基础URL
    base_url = 'https://support.okex.com'
    # 设置开始爬去的页面
    start_urls = ["https://support.okex.com/hc/zh-cn/sections/360000030652-%E6%9C%80%E6%96%B0%E5%85%AC%E5%91%8A"]
    # 自定义配置 pipeline
    custom_settings = {
        # 默认是DEBUG，即打印DEBUG, INFO, WARNING, ERROR，所有LOG信息
        'LOG_LEVEL': 'DEBUG',
        # 默认管道配置
        'ITEM_PIPELINES': {
            'spiders.pipelines.OkexPipeline.OkexPipeline': 1,
        },
        # 减慢爬取速度为1s
        'DOWNLOAD_DELAY': 1,
        # 指定redis数据库的连接参数
        'REDIS_HOST': '127.0.0.1',
        # 端口
        'REDIS_PORT': 6379,
        # 指定 redis链接密码，和使用哪一个数据库
        'REDIS_PARAMS': {
            'password': 'Aa123456',
            'db': 1,
        },
    }
    # 分类ID
    cate_id = 1

    def parse(self, response):
        sel = Selector(response)
        link = sel.xpath("//ul[@class='article-list']//a/@href").extract()
        for x in link:
            request = scrapy.Request(self.base_url + x, callback=self.parse_item)
            print(self.base_url + x)
            yield request

    def parse_item(self, response):
        item = ItemLoader(item=OkexItem(), response=response)
        item.add_xpath('title', "//header[@class='article-header']/h1/@title")
        item.add_xpath('desc', "//meta[@name='description']/@content")
        item.add_xpath('content', "//div[@class='article-body']")
        item.add_value('create_time', int(time.time()))
        item.add_value('source', parse.unquote(response.url))
        item.add_value('tag', '')
        item.add_value('author', "okex.com")
        item.add_value('click', '')
        item.add_value('cate_id', self.cate_id)
        return item.load_item()
